# Databricks notebook source
# MAGIC %run ../../global-setup $project_name=rag_chatbot

# COMMAND ----------

import pandas as pd 

df_evaluation = pd.read_csv("/Volumes/ml_in_action/rag_chatbot/files/Questions_Evaluation.csv").dropna().drop(columns=["id_question"])
df_eval = spark.createDataFrame(df_evaluation)
display(df_eval)

# COMMAND ----------

# MAGIC %sql
# MAGIC --Note that we need to enable Change Data Feed on the table to create the index
# MAGIC CREATE TABLE IF NOT EXISTS evaluation_table (
# MAGIC   id_question BIGINT GENERATED BY DEFAULT AS IDENTITY,
# MAGIC   question_asked STRING,
# MAGIC   answer_given STRING,
# MAGIC   pdf_name STRING,
# MAGIC   article_name STRING 
# MAGIC   ) TBLPROPERTIES (delta.enableChangeDataFeed = true); 

# COMMAND ----------

df_eval.write.mode("append").saveAsTable(f"{catalog}.{database_name}.evaluation_table")
